{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"name\": \"get_article_details\",\n",
      "  \"description\": \"Get article details from unstructured article text.\\ndate_published: formatted as \\\"MM/DD/YYYY\\\"\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"title\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"authors\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      },\n",
      "      \"short_summary\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"date_published\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"tags\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Article\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import inspect\n",
    "from typing import get_type_hints\n",
    "\n",
    "\n",
    "class Article:\n",
    "    pass\n",
    "\n",
    "\n",
    "class Weather:\n",
    "    pass\n",
    "\n",
    "\n",
    "class Directions:\n",
    "    pass\n",
    "\n",
    "\n",
    "def calculate_mortgage_payment(\n",
    "    loan_amount: int, interest_rate: float, loan_term: int\n",
    ") -> float:\n",
    "    \"\"\"Get the monthly mortgage payment given an interest rate percentage.\"\"\"\n",
    "\n",
    "    # TODO: you must implement this to actually call it later\n",
    "    pass\n",
    "\n",
    "\n",
    "def get_article_details(\n",
    "    title: str,\n",
    "    authors: list[str],\n",
    "    short_summary: str,\n",
    "    date_published: str,\n",
    "    tags: list[str],\n",
    ") -> Article:\n",
    "    '''Get article details from unstructured article text.\n",
    "    date_published: formatted as \"MM/DD/YYYY\"'''\n",
    "\n",
    "    # TODO: you must implement this to actually call it later\n",
    "    pass\n",
    "\n",
    "\n",
    "def get_weather(zip_code: str) -> Weather:\n",
    "    \"\"\"Get the current weather given a zip code.\"\"\"\n",
    "\n",
    "    # TODO: you must implement this to actually call it later\n",
    "    pass\n",
    "\n",
    "\n",
    "def get_directions(start: str, destination: str) -> Directions:\n",
    "    \"\"\"Get directions from Google Directions API.\n",
    "    start: start address as a string including zipcode (if any)\n",
    "    destination: end address as a string including zipcode (if any)\"\"\"\n",
    "\n",
    "    # TODO: you must implement this to actually call it later\n",
    "    pass\n",
    "\n",
    "\n",
    "def get_type_name(t):\n",
    "    name = str(t)\n",
    "    if \"list\" in name or \"dict\" in name:\n",
    "        return name\n",
    "    else:\n",
    "        return t.__name__\n",
    "\n",
    "\n",
    "def serialize_function_to_json(func):\n",
    "    signature = inspect.signature(func)\n",
    "    type_hints = get_type_hints(func)\n",
    "\n",
    "    function_info = {\n",
    "        \"name\": func.__name__,\n",
    "        \"description\": func.__doc__,\n",
    "        \"parameters\": {\"type\": \"object\", \"properties\": {}},\n",
    "        \"returns\": type_hints.get(\"return\", \"void\").__name__,\n",
    "    }\n",
    "\n",
    "    for name, _ in signature.parameters.items():\n",
    "        param_type = get_type_name(type_hints.get(name, type(None)))\n",
    "        function_info[\"parameters\"][\"properties\"][name] = {\"type\": param_type}\n",
    "\n",
    "    return json.dumps(function_info, indent=2)\n",
    "\n",
    "\n",
    "print(serialize_function_to_json(get_article_details))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import re\n",
    "\n",
    "\n",
    "def extract_function_calls(completion):\n",
    "    completion = completion.strip()\n",
    "    pattern = r\"(<multiplefunctions>(.*?)</multiplefunctions>)\"\n",
    "    match = re.search(pattern, completion, re.DOTALL)\n",
    "    if not match:\n",
    "        return None\n",
    "\n",
    "    multiplefn = match.group(1)\n",
    "    root = ET.fromstring(multiplefn)\n",
    "    functions = root.findall(\"functioncall\")\n",
    "    return [json.loads(fn.text) for fn in functions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_hermes_prompt(prompt, functions):\n",
    "    functions = \"\\n\\n\".join([serialize_function_to_json(fn) for fn in functions])\n",
    "    prompt = f\"\"\"<|im_start|>system\n",
    "You are a helpful assistant with access to the following functions:\n",
    "\n",
    "{functions}\n",
    "\n",
    "To use these functions respond with:\n",
    "<multiplefunctions>\n",
    "    <functioncall> {{\"name\": \"function_name\", \"arguments\": {{\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}}}} </functioncall>\n",
    "    <functioncall> {{\"name\": \"function_name\", \"arguments\": {{\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}}}} </functioncall>\n",
    "    ...\n",
    "</multiplefunctions>\n",
    "\n",
    "Edge cases you must handle:\n",
    "- If there are no functions that match the user request, you will respond politely that you cannot help.<|im_end|>\n",
    "<|im_start|>user\n",
    "{prompt}<|im_end|>\n",
    "<|im_start|>assistant\"\"\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|im_start|>system\n",
      "You are a helpful assistant with access to the following functions:\n",
      "\n",
      "{\n",
      "  \"name\": \"get_weather\",\n",
      "  \"description\": \"Get the current weather given a zip code.\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"zip_code\": {\n",
      "        \"type\": \"str\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Weather\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"name\": \"calculate_mortgage_payment\",\n",
      "  \"description\": \"Get the monthly mortgage payment given an interest rate percentage.\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"loan_amount\": {\n",
      "        \"type\": \"int\"\n",
      "      },\n",
      "      \"interest_rate\": {\n",
      "        \"type\": \"float\"\n",
      "      },\n",
      "      \"loan_term\": {\n",
      "        \"type\": \"int\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"float\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"name\": \"get_article_details\",\n",
      "  \"description\": \"Get article details from unstructured article text.\\ndate_published: formatted as \\\"MM/DD/YYYY\\\"\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"title\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"authors\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      },\n",
      "      \"short_summary\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"date_published\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"tags\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Article\"\n",
      "}\n",
      "\n",
      "To use these functions respond with:\n",
      "<multiplefunctions>\n",
      "    <functioncall> {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}} </functioncall>\n",
      "    <functioncall> {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}} </functioncall>\n",
      "    ...\n",
      "</multiplefunctions>\n",
      "\n",
      "Edge cases you must handle:\n",
      "- If there are no functions that match the user request, you will respond politely that you cannot help.<|im_end|>\n",
      "<|im_start|>user\n",
      "What's the weather in 10001?<|im_end|>\n",
      "<|im_start|>assistant\n",
      "<|im_start|>system\n",
      "You are a helpful assistant with access to the following functions:\n",
      "\n",
      "{\n",
      "  \"name\": \"get_weather\",\n",
      "  \"description\": \"Get the current weather given a zip code.\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"zip_code\": {\n",
      "        \"type\": \"str\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Weather\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"name\": \"calculate_mortgage_payment\",\n",
      "  \"description\": \"Get the monthly mortgage payment given an interest rate percentage.\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"loan_amount\": {\n",
      "        \"type\": \"int\"\n",
      "      },\n",
      "      \"interest_rate\": {\n",
      "        \"type\": \"float\"\n",
      "      },\n",
      "      \"loan_term\": {\n",
      "        \"type\": \"int\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"float\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"name\": \"get_article_details\",\n",
      "  \"description\": \"Get article details from unstructured article text.\\ndate_published: formatted as \\\"MM/DD/YYYY\\\"\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"title\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"authors\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      },\n",
      "      \"short_summary\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"date_published\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"tags\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Article\"\n",
      "}\n",
      "\n",
      "To use these functions respond with:\n",
      "<multiplefunctions>\n",
      "    <functioncall> {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}} </functioncall>\n",
      "    <functioncall> {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}} </functioncall>\n",
      "    ...\n",
      "</multiplefunctions>\n",
      "\n",
      "Edge cases you must handle:\n",
      "- If there are no functions that match the user request, you will respond politely that you cannot help.<|im_end|>\n",
      "<|im_start|>user\n",
      "Determine the monthly mortgage payment for a loan amount of $200,000, an interest rate of 4%, and a loan term of 30 years.<|im_end|>\n",
      "<|im_start|>assistant\n",
      "<|im_start|>system\n",
      "You are a helpful assistant with access to the following functions:\n",
      "\n",
      "{\n",
      "  \"name\": \"get_weather\",\n",
      "  \"description\": \"Get the current weather given a zip code.\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"zip_code\": {\n",
      "        \"type\": \"str\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Weather\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"name\": \"calculate_mortgage_payment\",\n",
      "  \"description\": \"Get the monthly mortgage payment given an interest rate percentage.\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"loan_amount\": {\n",
      "        \"type\": \"int\"\n",
      "      },\n",
      "      \"interest_rate\": {\n",
      "        \"type\": \"float\"\n",
      "      },\n",
      "      \"loan_term\": {\n",
      "        \"type\": \"int\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"float\"\n",
      "}\n",
      "\n",
      "{\n",
      "  \"name\": \"get_article_details\",\n",
      "  \"description\": \"Get article details from unstructured article text.\\ndate_published: formatted as \\\"MM/DD/YYYY\\\"\",\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"title\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"authors\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      },\n",
      "      \"short_summary\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"date_published\": {\n",
      "        \"type\": \"str\"\n",
      "      },\n",
      "      \"tags\": {\n",
      "        \"type\": \"list[str]\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"returns\": \"Article\"\n",
      "}\n",
      "\n",
      "To use these functions respond with:\n",
      "<multiplefunctions>\n",
      "    <functioncall> {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}} </functioncall>\n",
      "    <functioncall> {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_2\": value_2, ...}} </functioncall>\n",
      "    ...\n",
      "</multiplefunctions>\n",
      "\n",
      "Edge cases you must handle:\n",
      "- If there are no functions that match the user request, you will respond politely that you cannot help.<|im_end|>\n",
      "<|im_start|>user\n",
      "What's the current exchange rate for USD to EUR?<|im_end|>\n",
      "<|im_start|>assistant\n"
     ]
    }
   ],
   "source": [
    "prompts = [\n",
    "    \"What's the weather in 10001?\",\n",
    "    \"Determine the monthly mortgage payment for a loan amount of $200,000, an interest rate of 4%, and a loan term of 30 years.\",\n",
    "    \"What's the current exchange rate for USD to EUR?\",\n",
    "]\n",
    "functions = [get_weather, calculate_mortgage_payment, get_article_details]\n",
    "\n",
    "for prompt in prompts:\n",
    "    print(generate_hermes_prompt(prompt, functions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 2060, compute capability 7.5\n",
      "llama_model_loader: loaded meta data with 20 key-value pairs and 291 tensors from ../../models/OpenHermes-2.5-Mistral-7B-GGUF/openhermes-2.5-mistral-7b.Q4_K_M.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_K     [  4096, 32002,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:              blk.0.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:              blk.0.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:         blk.0.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:            blk.0.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:              blk.0.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:              blk.1.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:              blk.1.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:         blk.1.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:            blk.1.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:              blk.1.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:              blk.2.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:              blk.2.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:         blk.2.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:            blk.2.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:              blk.2.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:              blk.3.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:              blk.3.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:         blk.3.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:            blk.3.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:              blk.3.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:              blk.4.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:              blk.4.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:         blk.4.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:            blk.4.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:              blk.4.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:              blk.5.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:              blk.5.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:         blk.5.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:            blk.5.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:              blk.5.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:              blk.6.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:              blk.6.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:         blk.6.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:            blk.6.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:              blk.6.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:              blk.7.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:              blk.7.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:         blk.7.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:            blk.7.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:              blk.7.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:              blk.8.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:              blk.8.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:         blk.8.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:            blk.8.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:              blk.8.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:              blk.9.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:              blk.9.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:         blk.9.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:            blk.9.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:              blk.9.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:             blk.10.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:             blk.10.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:        blk.10.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:           blk.10.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:             blk.10.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:             blk.11.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:             blk.11.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:        blk.11.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:           blk.11.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:             blk.11.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:             blk.12.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:             blk.12.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:        blk.12.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:           blk.12.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:             blk.12.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:             blk.13.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:             blk.13.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:        blk.13.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:           blk.13.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:             blk.13.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:             blk.14.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:             blk.14.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:        blk.14.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:           blk.14.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:             blk.14.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:             blk.15.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:             blk.15.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:        blk.15.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:           blk.15.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:             blk.15.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:             blk.16.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:             blk.16.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:        blk.16.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:           blk.16.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:             blk.16.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:             blk.17.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:             blk.17.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:        blk.17.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:           blk.17.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:             blk.17.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:             blk.18.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:             blk.18.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:        blk.18.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:           blk.18.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:             blk.18.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:             blk.19.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:             blk.19.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:        blk.19.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:           blk.19.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:             blk.19.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:             blk.20.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:             blk.20.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:        blk.20.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:           blk.20.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:             blk.20.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:             blk.21.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:             blk.21.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:        blk.21.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:           blk.21.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:             blk.21.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:             blk.22.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:             blk.22.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:        blk.22.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:           blk.22.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:             blk.22.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:             blk.23.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:             blk.23.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:        blk.23.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:           blk.23.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:             blk.23.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:             blk.24.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:             blk.24.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:        blk.24.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:           blk.24.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:             blk.24.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:             blk.25.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:             blk.25.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:        blk.25.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:           blk.25.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:             blk.25.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:             blk.26.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:             blk.26.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_v.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:        blk.26.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:           blk.26.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:             blk.26.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_down.weight q4_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:             blk.27.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:             blk.27.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:        blk.27.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:           blk.27.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:             blk.27.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:             blk.28.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:             blk.28.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:        blk.28.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:           blk.28.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:             blk.28.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:             blk.29.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:             blk.29.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:        blk.29.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:           blk.29.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:             blk.29.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:             blk.30.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:             blk.30.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:        blk.30.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:           blk.30.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:             blk.30.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:             blk.31.attn_q.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:             blk.31.attn_k.weight q4_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_v.weight q6_K     [  4096,  1024,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:        blk.31.attn_output.weight q4_K     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:           blk.31.ffn_gate.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:             blk.31.ffn_up.weight q4_K     [  4096, 14336,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_down.weight q6_K     [ 14336,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:                    output.weight q6_K     [  4096, 32002,     1,     1 ]\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = teknium_openhermes-2.5-mistral-7b\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 32768\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                       llama.rope.freq_base f32              = 10000.000000\n",
      "llama_model_loader: - kv  11:                          general.file_type u32              = 15\n",
      "llama_model_loader: - kv  12:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.tokens arr[str,32002]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  14:                      tokenizer.ggml.scores arr[f32,32002]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  15:                  tokenizer.ggml.token_type arr[i32,32002]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 32000\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.padding_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 261/32002 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32002\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 32768\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 7.24 B\n",
      "llm_load_print_meta: model size       = 4.07 GiB (4.83 BPW) \n",
      "llm_load_print_meta: general.name   = teknium_openhermes-2.5-mistral-7b\n",
      "llm_load_print_meta: BOS token = 1 '<s>'\n",
      "llm_load_print_meta: EOS token = 32000 '<|im_end|>'\n",
      "llm_load_print_meta: UNK token = 0 '<unk>'\n",
      "llm_load_print_meta: PAD token = 0 '<unk>'\n",
      "llm_load_print_meta: LF token  = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: mem required  =   70.42 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 35/35 layers to GPU\n",
      "llm_load_tensors: VRAM used: 4095.06 MiB\n",
      "...............................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 2048\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: offloading v cache to GPU\n",
      "llama_kv_cache_init: offloading k cache to GPU\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MiB\n",
      "llama_new_context_with_model: kv self size  =  256.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 740/740\n",
      "llama_new_context_with_model: compute buffer total size = 159.07 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 156.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 4507.07 MiB (model: 4095.06 MiB, context: 412.00 MiB)\n"
     ]
    }
   ],
   "source": [
    "import llama_cpp\n",
    "\n",
    "llama = llama_cpp.Llama(\n",
    "    model_path=\"../../models/OpenHermes-2.5-Mistral-7B-GGUF/openhermes-2.5-mistral-7b.Q4_K_M.gguf\",\n",
    "    n_gpu_layers=-1,\n",
    "    n_ctx=2048,\n",
    "    verbose=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'name': 'get_weather', 'arguments': {'zip_code': '10001'}}]\n",
      "====================================================================================================\n",
      "[{'name': 'calculate_mortgage_payment', 'arguments': {'loan_amount': 200000, 'interest_rate': 0.04, 'loan_term': 30}}]\n",
      "====================================================================================================\n",
      "Unfortunately, I do not have a built-in function to check currency exchange rates. However, you can use third-party APIs or websites like Google Finance or XE to get this information.\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "prompts = [\n",
    "    \"What's the weather in 10001?\",\n",
    "    \"Determine the monthly mortgage payment for a loan amount of $200,000, an interest rate of 4%, and a loan term of 30 years.\",\n",
    "    \"What's the current exchange rate for USD to EUR?\",\n",
    "]\n",
    "functions = [get_weather, calculate_mortgage_payment, get_article_details]\n",
    "\n",
    "for prompt in prompts:\n",
    "    prompt = generate_hermes_prompt(prompt, functions)\n",
    "    completion = llama.create_completion(prompt, max_tokens=-1)[\"choices\"][0][\"text\"]\n",
    "    function_calls = extract_function_calls(completion)\n",
    "    if function_calls:\n",
    "        print(function_calls)\n",
    "    else:\n",
    "        print(completion.strip())\n",
    "    print(\"=\" * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get_weather\n",
      "{'zip_code': '05751'}\n",
      "====================================================================================================\n",
      "get_weather\n",
      "{'zip_code': '05751'}\n",
      "get_weather\n",
      "{'zip_code': '07030'}\n",
      "calculate_mortgage_payment\n",
      "{'loan_amount': 250000, 'interest_rate': 4.18, 'loan_term': 30}\n",
      "====================================================================================================\n",
      "I don't have a function to get exchange rates, but I can provide some resources where you can find this information. You can check websites like Google Finance, XE.com, or Yahoo Finance for up-to-date currency exchange rates.\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "prompts = [\n",
    "    \"What's the weather in 05751?\",\n",
    "    \"I'm planning a trip to Killington, Vermont (05751) from Hoboken, NJ (07030). Can you get me weather for both locations and directions?\",\n",
    "    \"What's the current exchange rate for USD to EUR?\",\n",
    "]\n",
    "\n",
    "for prompt in prompts:\n",
    "    completion = llama.create_completion(\n",
    "        generate_hermes_prompt(prompt, functions), max_tokens=-1\n",
    "    )[\"choices\"][0][\"text\"]\n",
    "    function_calls = extract_function_calls(completion)\n",
    "\n",
    "    if function_calls:\n",
    "        for function in function_calls:\n",
    "            print(function[\"name\"])\n",
    "            print(function[\"arguments\"])\n",
    "    else:\n",
    "        print(completion.strip())\n",
    "\n",
    "    print(\"=\" * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
